{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bert_movie_reviews.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "90jnBysZe4AI"
      ],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "68015c8021da4c1d91f66e42c5fef8ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e4ef62149f1e4d179f94bd6431a4c975",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b321d98b02cd41969b95c73c6ea0de29",
              "IPY_MODEL_5aef7a7ab6684b7ead435005321addee"
            ]
          }
        },
        "e4ef62149f1e4d179f94bd6431a4c975": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b321d98b02cd41969b95c73c6ea0de29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7728c0d5e9cc4011952fe7a04faf1f9a",
            "_dom_classes": [],
            "description": "Dl Completed...: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_15312aa4d0d845ae86cc44e7a7c86d8f"
          }
        },
        "5aef7a7ab6684b7ead435005321addee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e43b9bc4e70e488a9f9eedaf1a9c61c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:04&lt;00:00,  4.42s/ url]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de1cbaf3dd9e4ed2ad6f4d8deda561ad"
          }
        },
        "7728c0d5e9cc4011952fe7a04faf1f9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "15312aa4d0d845ae86cc44e7a7c86d8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e43b9bc4e70e488a9f9eedaf1a9c61c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de1cbaf3dd9e4ed2ad6f4d8deda561ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eaf8e5945ec24c3ca2ac6bb1833af20a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dc6c8baee68f4bae8192a0f53153e3ea",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d60b6d023c724653a1b9883295765323",
              "IPY_MODEL_2a9376fa1b6841d3b09e8f23331bc7f5"
            ]
          }
        },
        "dc6c8baee68f4bae8192a0f53153e3ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d60b6d023c724653a1b9883295765323": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_60e25b7765da4e2da7f14fa0e33263a5",
            "_dom_classes": [],
            "description": "Dl Size...: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e1fb6461285244508b3da98e297b7c86"
          }
        },
        "2a9376fa1b6841d3b09e8f23331bc7f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d05570d734e4446b8b2e3c9b38e80d88",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 80/80 [00:04&lt;00:00, 18.24 MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_64524aa8a718466382a580f9d11f8ba6"
          }
        },
        "60e25b7765da4e2da7f14fa0e33263a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e1fb6461285244508b3da98e297b7c86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d05570d734e4446b8b2e3c9b38e80d88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "64524aa8a718466382a580f9d11f8ba6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "47484fef445b495c90f4778d1851c76d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_40ab14f225de47c4b132323cc83920d4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2b1409dfc00a4224849f9690f812e272",
              "IPY_MODEL_bdd98dfb0398424c95fd74fb2cd18791"
            ]
          }
        },
        "40ab14f225de47c4b132323cc83920d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2b1409dfc00a4224849f9690f812e272": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ac1d0169ee7e4bb19c94f8a3cdf6ac82",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_60401ac690a14c92bc71c9b71b6dd73a"
          }
        },
        "bdd98dfb0398424c95fd74fb2cd18791": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7e6a478876e2467383d04bf243b89631",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25000/0 [00:17&lt;00:00, 2364.87 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1994d79a1f184ea484f108fe5675eaf4"
          }
        },
        "ac1d0169ee7e4bb19c94f8a3cdf6ac82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "60401ac690a14c92bc71c9b71b6dd73a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7e6a478876e2467383d04bf243b89631": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1994d79a1f184ea484f108fe5675eaf4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf2949f80c32416cad1d38758ef236bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_014d9e7cb7524fd690851d7689423b3e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f2c125388b4e4d7d960ecb3cb1de3668",
              "IPY_MODEL_c958e3d051a34e4a82a4f46b0e0b72fa"
            ]
          }
        },
        "014d9e7cb7524fd690851d7689423b3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f2c125388b4e4d7d960ecb3cb1de3668": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b983834e3a164464a3b794a3495774e7",
            "_dom_classes": [],
            "description": " 44%",
            "_model_name": "IntProgressModel",
            "bar_style": "danger",
            "max": 25000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 11076,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18828e8e48304901bae64a13afe046cf"
          }
        },
        "c958e3d051a34e4a82a4f46b0e0b72fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bda87ce07d8c4fd7949de7135a207ae0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 11076/25000 [00:00&lt;00:00, 110754.03 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9b462f3739ff41578693c8cf2dfcfbb0"
          }
        },
        "b983834e3a164464a3b794a3495774e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18828e8e48304901bae64a13afe046cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bda87ce07d8c4fd7949de7135a207ae0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9b462f3739ff41578693c8cf2dfcfbb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61a4679998634298884ba0ba92b8ee37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_837c80675a8e490a8893407f5ca87127",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7fcd136ce11946e0a4422fe7f491794f",
              "IPY_MODEL_900349441b374608ad9f9ca9e38d469c"
            ]
          }
        },
        "837c80675a8e490a8893407f5ca87127": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7fcd136ce11946e0a4422fe7f491794f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_60ac84b64aac48218ca3d240bf6e33ee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1dc7e072071493ca782a35b2e5080bf"
          }
        },
        "900349441b374608ad9f9ca9e38d469c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e917fa6503444eb9906651011b467508",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25000/0 [00:17&lt;00:00, 2402.56 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_72bd885d726743498aead4770d89c2a4"
          }
        },
        "60ac84b64aac48218ca3d240bf6e33ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1dc7e072071493ca782a35b2e5080bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e917fa6503444eb9906651011b467508": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "72bd885d726743498aead4770d89c2a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "29b58d83192140f4aedfb4557673a096": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2ff6fca755314d51904b82cf43d44691",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2e9ddd8f4ea9400b9b962aafd10e46ff",
              "IPY_MODEL_0c487c8cf83142e7b6e96d7e2e2e33e3"
            ]
          }
        },
        "2ff6fca755314d51904b82cf43d44691": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2e9ddd8f4ea9400b9b962aafd10e46ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_57001dffac864d039a8e455b1a475609",
            "_dom_classes": [],
            "description": " 46%",
            "_model_name": "IntProgressModel",
            "bar_style": "danger",
            "max": 25000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 11615,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e71fb07b1dde4f0f93a44c9777872d1c"
          }
        },
        "0c487c8cf83142e7b6e96d7e2e2e33e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_92453812abf045eba5fa6800495a7681",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 11615/25000 [00:00&lt;00:00, 116147.34 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a811395ea54741a284bc15ea2d113c05"
          }
        },
        "57001dffac864d039a8e455b1a475609": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e71fb07b1dde4f0f93a44c9777872d1c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "92453812abf045eba5fa6800495a7681": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a811395ea54741a284bc15ea2d113c05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f55802bda434cb4b27690554a9d22a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d0353b16bf0b4ea8941ef678e7e40ebe",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d1be15b5c4d74c88b173a00765b59510",
              "IPY_MODEL_93dc558e1bb5470c9195e3cce03d342a"
            ]
          }
        },
        "d0353b16bf0b4ea8941ef678e7e40ebe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d1be15b5c4d74c88b173a00765b59510": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2f505dd090f8454db8b53ae58d46a866",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89a1d7ab789c4324900c2cddffab4348"
          }
        },
        "93dc558e1bb5470c9195e3cce03d342a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c9f0562529264b68b47d1fe36d6f385b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 50000/0 [00:25&lt;00:00, 2427.74 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ddba144045664f89b2ae87d13dfc00f8"
          }
        },
        "2f505dd090f8454db8b53ae58d46a866": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89a1d7ab789c4324900c2cddffab4348": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9f0562529264b68b47d1fe36d6f385b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ddba144045664f89b2ae87d13dfc00f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e0830eff6f4c4852a07dc7dc1fd91a2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a26b89532b5c48d595e7e15f889e161a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ce6a47a2e83249eca974e8b13a8b4964",
              "IPY_MODEL_0bcd86588b47459eb1bb67a13484dc52"
            ]
          }
        },
        "a26b89532b5c48d595e7e15f889e161a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce6a47a2e83249eca974e8b13a8b4964": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_73025cdb841d4c0d8e58705e7065c1b1",
            "_dom_classes": [],
            "description": " 60%",
            "_model_name": "IntProgressModel",
            "bar_style": "danger",
            "max": 50000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 30232,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_28c5f2ebeef14421b8989c5895489009"
          }
        },
        "0bcd86588b47459eb1bb67a13484dc52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_046be065316f4782abd36ad116feb782",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 30232/50000 [00:07&lt;00:00, 96749.64 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_982c08cd3ab34e8aaa3426e0c2650a62"
          }
        },
        "73025cdb841d4c0d8e58705e7065c1b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "28c5f2ebeef14421b8989c5895489009": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "046be065316f4782abd36ad116feb782": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "982c08cd3ab34e8aaa3426e0c2650a62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90jnBysZe4AI",
        "colab_type": "text"
      },
      "source": [
        "### Requirements (for Colab users)\n",
        "- Follow these instructions to install requirements if you are using Colab\n",
        "- Then, restart runtime and skip this section"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aU3YLZ1TYKUt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q tensorflow"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k8c5OBk8eVUL",
        "colab_type": "code",
        "outputId": "b22dad65-b555-44f3-848e-d3446d33a93e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 155
        }
      },
      "source": [
        "!git clone https://github.com/tensorflow/models.git"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'models'...\n",
            "remote: Enumerating objects: 50, done.\u001b[K\n",
            "remote: Counting objects:   2% (1/50)\u001b[K\rremote: Counting objects:   4% (2/50)\u001b[K\rremote: Counting objects:   6% (3/50)\u001b[K\rremote: Counting objects:   8% (4/50)\u001b[K\rremote: Counting objects:  10% (5/50)\u001b[K\rremote: Counting objects:  12% (6/50)\u001b[K\rremote: Counting objects:  14% (7/50)\u001b[K\rremote: Counting objects:  16% (8/50)\u001b[K\rremote: Counting objects:  18% (9/50)\u001b[K\rremote: Counting objects:  20% (10/50)\u001b[K\rremote: Counting objects:  22% (11/50)\u001b[K\rremote: Counting objects:  24% (12/50)\u001b[K\rremote: Counting objects:  26% (13/50)\u001b[K\rremote: Counting objects:  28% (14/50)\u001b[K\rremote: Counting objects:  30% (15/50)\u001b[K\rremote: Counting objects:  32% (16/50)\u001b[K\rremote: Counting objects:  34% (17/50)\u001b[K\rremote: Counting objects:  36% (18/50)\u001b[K\rremote: Counting objects:  38% (19/50)\u001b[K\rremote: Counting objects:  40% (20/50)\u001b[K\rremote: Counting objects:  42% (21/50)\u001b[K\rremote: Counting objects:  44% (22/50)\u001b[K\rremote: Counting objects:  46% (23/50)\u001b[K\rremote: Counting objects:  48% (24/50)\u001b[K\rremote: Counting objects:  50% (25/50)\u001b[K\rremote: Counting objects:  52% (26/50)\u001b[K\rremote: Counting objects:  54% (27/50)\u001b[K\rremote: Counting objects:  56% (28/50)\u001b[K\rremote: Counting objects:  58% (29/50)\u001b[K\rremote: Counting objects:  60% (30/50)\u001b[K\rremote: Counting objects:  62% (31/50)\u001b[K\rremote: Counting objects:  64% (32/50)\u001b[K\rremote: Counting objects:  66% (33/50)\u001b[K\rremote: Counting objects:  68% (34/50)\u001b[K\rremote: Counting objects:  70% (35/50)\u001b[K\rremote: Counting objects:  72% (36/50)\u001b[K\rremote: Counting objects:  74% (37/50)\u001b[K\rremote: Counting objects:  76% (38/50)\u001b[K\rremote: Counting objects:  78% (39/50)\u001b[K\rremote: Counting objects:  80% (40/50)\u001b[K\rremote: Counting objects:  82% (41/50)\u001b[K\rremote: Counting objects:  84% (42/50)\u001b[K\rremote: Counting objects:  86% (43/50)\u001b[K\rremote: Counting objects:  88% (44/50)\u001b[K\rremote: Counting objects:  90% (45/50)\u001b[K\rremote: Counting objects:  92% (46/50)\u001b[K\rremote: Counting objects:  94% (47/50)\u001b[K\rremote: Counting objects:  96% (48/50)\u001b[K\rremote: Counting objects:  98% (49/50)\u001b[K\rremote: Counting objects: 100% (50/50)\u001b[K\rremote: Counting objects: 100% (50/50), done.\u001b[K\n",
            "remote: Compressing objects: 100% (48/48), done.\u001b[K\n",
            "remote: Total 33134 (delta 10), reused 37 (delta 2), pack-reused 33084\u001b[K\n",
            "Receiving objects: 100% (33134/33134), 511.97 MiB | 41.34 MiB/s, done.\n",
            "Resolving deltas: 100% (21222/21222), done.\n",
            "Checking out files: 100% (2466/2466), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3H2G0571zLLs",
        "colab_type": "code",
        "outputId": "aee0ed54-0e1e-4c2a-fd18-2f9a4842a0db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        }
      },
      "source": [
        "# install requirements to use tensorflow/models repository\n",
        "!pip install --user -q -r models/official/requirements.txt\n",
        "# you may have to restart the runtime afterwards"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 92kB 4.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 102kB 5.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.0MB 45.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 21.6MB 153kB/s \n",
            "\u001b[?25h  Building wheel for py-cpuinfo (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[33m  WARNING: The script cpuinfo is installed in '/root/.local/bin' which is not on PATH.\n",
            "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IMsEoT3Fg4Wg",
        "colab_type": "text"
      },
      "source": [
        "## data loading and preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GmqEylyFYTdP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import tensorflow_datasets as tfds\n",
        "import tensorflow_hub as hub\n",
        "import sys\n",
        "sys.path.append('models')\n",
        "from official.nlp.data import classifier_data_lib\n",
        "from official.nlp.bert import tokenization\n",
        "from official.nlp import optimization"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZuX1lB8pPJ-W",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "1a236840-45e6-40dc-f715-da73f0502271"
      },
      "source": [
        "print(\"TF Version: \", tf.__version__)\n",
        "print(\"Eager mode: \", tf.executing_eagerly())\n",
        "print(\"Hub version: \", hub.__version__)\n",
        "print(\"GPU is\", \"available\" if tf.config.experimental.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TF Version:  2.2.0-rc1\n",
            "Eager mode:  True\n",
            "Hub version:  0.7.0\n",
            "GPU is available\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "67pOyI6CPiPK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 353,
          "referenced_widgets": [
            "68015c8021da4c1d91f66e42c5fef8ce",
            "e4ef62149f1e4d179f94bd6431a4c975",
            "b321d98b02cd41969b95c73c6ea0de29",
            "5aef7a7ab6684b7ead435005321addee",
            "7728c0d5e9cc4011952fe7a04faf1f9a",
            "15312aa4d0d845ae86cc44e7a7c86d8f",
            "e43b9bc4e70e488a9f9eedaf1a9c61c9",
            "de1cbaf3dd9e4ed2ad6f4d8deda561ad",
            "eaf8e5945ec24c3ca2ac6bb1833af20a",
            "dc6c8baee68f4bae8192a0f53153e3ea",
            "d60b6d023c724653a1b9883295765323",
            "2a9376fa1b6841d3b09e8f23331bc7f5",
            "60e25b7765da4e2da7f14fa0e33263a5",
            "e1fb6461285244508b3da98e297b7c86",
            "d05570d734e4446b8b2e3c9b38e80d88",
            "64524aa8a718466382a580f9d11f8ba6",
            "47484fef445b495c90f4778d1851c76d",
            "40ab14f225de47c4b132323cc83920d4",
            "2b1409dfc00a4224849f9690f812e272",
            "bdd98dfb0398424c95fd74fb2cd18791",
            "ac1d0169ee7e4bb19c94f8a3cdf6ac82",
            "60401ac690a14c92bc71c9b71b6dd73a",
            "7e6a478876e2467383d04bf243b89631",
            "1994d79a1f184ea484f108fe5675eaf4",
            "cf2949f80c32416cad1d38758ef236bf",
            "014d9e7cb7524fd690851d7689423b3e",
            "f2c125388b4e4d7d960ecb3cb1de3668",
            "c958e3d051a34e4a82a4f46b0e0b72fa",
            "b983834e3a164464a3b794a3495774e7",
            "18828e8e48304901bae64a13afe046cf",
            "bda87ce07d8c4fd7949de7135a207ae0",
            "9b462f3739ff41578693c8cf2dfcfbb0",
            "61a4679998634298884ba0ba92b8ee37",
            "837c80675a8e490a8893407f5ca87127",
            "7fcd136ce11946e0a4422fe7f491794f",
            "900349441b374608ad9f9ca9e38d469c",
            "60ac84b64aac48218ca3d240bf6e33ee",
            "c1dc7e072071493ca782a35b2e5080bf",
            "e917fa6503444eb9906651011b467508",
            "72bd885d726743498aead4770d89c2a4",
            "29b58d83192140f4aedfb4557673a096",
            "2ff6fca755314d51904b82cf43d44691",
            "2e9ddd8f4ea9400b9b962aafd10e46ff",
            "0c487c8cf83142e7b6e96d7e2e2e33e3",
            "57001dffac864d039a8e455b1a475609",
            "e71fb07b1dde4f0f93a44c9777872d1c",
            "92453812abf045eba5fa6800495a7681",
            "a811395ea54741a284bc15ea2d113c05",
            "4f55802bda434cb4b27690554a9d22a1",
            "d0353b16bf0b4ea8941ef678e7e40ebe",
            "d1be15b5c4d74c88b173a00765b59510",
            "93dc558e1bb5470c9195e3cce03d342a",
            "2f505dd090f8454db8b53ae58d46a866",
            "89a1d7ab789c4324900c2cddffab4348",
            "c9f0562529264b68b47d1fe36d6f385b",
            "ddba144045664f89b2ae87d13dfc00f8",
            "e0830eff6f4c4852a07dc7dc1fd91a2a",
            "a26b89532b5c48d595e7e15f889e161a",
            "ce6a47a2e83249eca974e8b13a8b4964",
            "0bcd86588b47459eb1bb67a13484dc52",
            "73025cdb841d4c0d8e58705e7065c1b1",
            "28c5f2ebeef14421b8989c5895489009",
            "046be065316f4782abd36ad116feb782",
            "982c08cd3ab34e8aaa3426e0c2650a62"
          ]
        },
        "outputId": "ccb4884f-45d1-4308-fd1c-964fc9a218ae"
      },
      "source": [
        "train_data, test_data = tfds.load(\n",
        "    name=\"imdb_reviews\", \n",
        "    split=('train', 'test'),\n",
        "    as_supervised=True)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDownloading and preparing dataset imdb_reviews/plain_text/1.0.0 (download: 80.23 MiB, generated: Unknown size, total: 80.23 MiB) to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0...\u001b[0m\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "68015c8021da4c1d91f66e42c5fef8ce",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', description='Dl Completed...', max=1, style=ProgressStyl…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eaf8e5945ec24c3ca2ac6bb1833af20a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', description='Dl Size...', max=1, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "47484fef445b495c90f4778d1851c76d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rShuffling and writing examples to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0.incomplete03J32D/imdb_reviews-train.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cf2949f80c32416cad1d38758ef236bf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=25000), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61a4679998634298884ba0ba92b8ee37",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rShuffling and writing examples to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0.incomplete03J32D/imdb_reviews-test.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "29b58d83192140f4aedfb4557673a096",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=25000), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4f55802bda434cb4b27690554a9d22a1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rShuffling and writing examples to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0.incomplete03J32D/imdb_reviews-unsupervised.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e0830eff6f4c4852a07dc7dc1fd91a2a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=50000), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDataset imdb_reviews downloaded and prepared to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0. Subsequent calls will reuse this data.\u001b[0m\n",
            "\r"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bnhUDyOSm45X",
        "colab_type": "code",
        "outputId": "ec2c87cb-962a-4b46-da16-2ab5f6866286",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "# a sample of data\n",
        "for text, label in train_data.take(1):\n",
        "  print(text)\n",
        "  print(label)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(b\"This was an absolutely terrible movie. Don't be lured in by Christopher Walken or Michael Ironside. Both are great actors, but this must simply be their worst role in history. Even their great acting could not redeem this movie's ridiculous storyline. This movie is an early nineties US propaganda piece. The most pathetic scenes were those when the Columbian rebels were making their cases for revolutions. Maria Conchita Alonso appeared phony, and her pseudo-love affair with Walken was nothing but a pathetic emotional plug in a movie that was devoid of any real meaning. I am disappointed that there are movies like this, ruining actor's like Christopher Walken's good name. I could barely sit through it.\", shape=(), dtype=string)\n",
            "tf.Tensor(0, shape=(), dtype=int64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EMb5M86b4-BU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"\n",
        "Each line of the dataset is composed of the review text and its label\n",
        "- Data preprocessing consists of transforming text to BERT input features:\n",
        "input_word_ids, input_mask, segment_ids\n",
        "- In the process, tokenizing the text is done with the provided BERT model tokenizer\n",
        "\"\"\"\n",
        "\n",
        "label_list = [0, 1] # Label categories\n",
        "max_seq_length = 128 # maximum length of (token) input sequences\n",
        "train_batch_size = 32\n",
        "\n",
        "# Get BERT layer and tokenizer:\n",
        "# More details here: https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\n",
        "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n",
        "                            trainable=True)\n",
        "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
        "tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n",
        "\n",
        "\n",
        "# This provides a function to convert row to input features and label\n",
        "def to_feature_map_fn(label_list, max_seq_length, tokenizer):\n",
        "  def to_feature(text, label):\n",
        "    example = classifier_data_lib.InputExample(guid = None,\n",
        "                                              text_a = text.numpy(), \n",
        "                                              text_b = None, \n",
        "                                              label = label.numpy())\n",
        "    feature = classifier_data_lib.convert_single_example(5, example, label_list,\n",
        "                                      max_seq_length, tokenizer)\n",
        "    \n",
        "    return (feature.input_ids, feature.input_mask, feature.segment_ids, feature.label_id)\n",
        "\n",
        "  def to_feature_map(text, label):\n",
        "    input_ids, input_mask, segment_ids, label_id = tf.py_function(to_feature, inp=[text, label], \n",
        "                                  Tout=[tf.int32, tf.int32, tf.int32, tf.int32])\n",
        "  \n",
        "    # py_func doesn't set the shape of the returned tensors.\n",
        "    input_ids.set_shape([128])\n",
        "    input_mask.set_shape([128])\n",
        "    segment_ids.set_shape([128])\n",
        "    label_id.set_shape([])\n",
        "\n",
        "    x = {\n",
        "          'input_word_ids': input_ids,\n",
        "          'input_mask': input_mask,\n",
        "          'input_type_ids': segment_ids\n",
        "      }\n",
        "    return (x, label_id)\n",
        "  return to_feature_map\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LHRdiO3dnPNr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "to_feature_map = to_feature_map_fn(label_list, max_seq_length, tokenizer)\n",
        "\n",
        "# train\n",
        "train_data = (train_data.map(to_feature_map,\n",
        "                            num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "                        .shuffle(100)\n",
        "                        .batch(32, drop_remainder=True)\n",
        "                        .prefetch(tf.data.experimental.AUTOTUNE))\n",
        "\n",
        "# test\n",
        "test_data = (test_data.map(to_feature_map,\n",
        "                          num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "                        .batch(32, drop_remainder=True)\n",
        "                        .prefetch(tf.data.experimental.AUTOTUNE))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0Z2cy9GHQ8x",
        "colab_type": "code",
        "outputId": "5ca75fb1-d8ce-4961-c044-3c4b1f4f31ce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "# data spec\n",
        "train_data.element_spec"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'input_mask': TensorSpec(shape=(32, 128), dtype=tf.int32, name=None),\n",
              "  'input_type_ids': TensorSpec(shape=(32, 128), dtype=tf.int32, name=None),\n",
              "  'input_word_ids': TensorSpec(shape=(32, 128), dtype=tf.int32, name=None)},\n",
              " TensorSpec(shape=(32,), dtype=tf.int32, name=None))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GZxe-7yhPyQe",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9il4gtlADcp",
        "colab_type": "code",
        "outputId": "f0aa8b17-5bce-4d78-8d50-407102500b38",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "source": [
        "# Building the model\n",
        "\n",
        "tf.keras.backend.clear_session()\n",
        "\n",
        "input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
        "                                       name=\"input_word_ids\")\n",
        "input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
        "                                   name=\"input_mask\")\n",
        "input_type_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
        "                                    name=\"input_type_ids\")\n",
        "\n",
        "pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, input_type_ids])\n",
        "\n",
        "output = tf.keras.layers.Dense(2, activation='softmax', name=\"output\")(pooled_output)\n",
        "#drop = tf.keras.layers.Dropout(0.5)(pooled_output)\n",
        "\n",
        "model = tf.keras.Model(\n",
        "      inputs={\n",
        "          'input_word_ids': input_word_ids,\n",
        "          'input_mask': input_mask,\n",
        "          'input_type_ids': input_type_ids\n",
        "      },\n",
        "      outputs=output)\n",
        "model.summary()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_word_ids (InputLayer)     [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_mask (InputLayer)         [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_type_ids (InputLayer)     [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "keras_layer (KerasLayer)        [(None, 768), (None, 109482241   input_word_ids[0][0]             \n",
            "                                                                 input_mask[0][0]                 \n",
            "                                                                 input_type_ids[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "output (Dense)                  (None, 2)            1538        keras_layer[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 109,483,779\n",
            "Trainable params: 109,483,778\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S6maM-vr7YaJ",
        "colab_type": "text"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ptCtiiONsBgo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer=tf.keras.optimizers.Adam(learning_rate=2e-5)\n",
        "model.compile(optimizer=optimizer, \n",
        "              loss='sparse_categorical_crossentropy', \n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OcREcgPUHr9O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "45a62d57-7de9-4e3c-a6a7-8b4f3d9057e4"
      },
      "source": [
        "# Train model\n",
        "epochs = 3\n",
        "history = model.fit(train_data,\n",
        "                    epochs=epochs,\n",
        "                    verbose=1)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "781/781 [==============================] - 344s 441ms/step - loss: 0.3240 - accuracy: 0.8552\n",
            "Epoch 2/3\n",
            "781/781 [==============================] - 344s 441ms/step - loss: 0.1384 - accuracy: 0.9488\n",
            "Epoch 3/3\n",
            "781/781 [==============================] - 344s 440ms/step - loss: 0.0484 - accuracy: 0.9849\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hkhtCCgnUbY6",
        "colab_type": "code",
        "outputId": "e97912ae-f78d-4862-921e-9546fecef038",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "model.evaluate(test_data, verbose=1)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "781/781 [==============================] - 138s 176ms/step - loss: 0.4277 - accuracy: 0.8770\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.4276823401451111, 0.877000629901886]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    }
  ]
}